%Porter Stemmer Test

%% initialize all data, features
clear; close all; clc; 

load ../../data/music_dataset.mat
addpath '../CV/' '../DT/' '../'

[Xt_lyrics] = make_lyrics_sparse(train, vocab);
[Xq_lyrics] = make_lyrics_sparse(quiz, vocab);

Yt = zeros(numel(train), 1);
for i=1:numel(train)
    Yt(i) = genre_class(train(i).genre);
end

Xt_audio = make_audio(train);
Xq_audio = make_audio(quiz);

%% use porter stemmer to combine/reduce lyrics features
tic
%create a cell array of stems corresponding to the vocab array
stems = cell(1,size(vocab,2));

for i=1:size(vocab,2)
    stems{i} = porterStemmer(vocab{i});
end

%% now find similar stems and combine their features in Xt_lyrics
i=1;
ustems = unique(lower(stems));
Xt_lyrics_stemmed = zeros(size(Xt_lyrics,1),numel(ustems));

for i=1:numel(ustems)
    
    %find indices of all words that have same stem
    l = strcmpi(ustems{i},stems);  
    idx = find(l);
    
    %combine those features into one feature vector in the new Xt
    Xt_lyrics_stemmed(:,i) = sum(full(Xt_lyrics(:,idx)),2);  
    
end

%make sparse
Xt_lyrics_stemmed = sparse(Xt_lyrics_stemmed);
toc
